/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package com.wsc.indexer;

/**
 *
 * @author Madhu
 */



import java.io.UnsupportedEncodingException;
import java.security.NoSuchAlgorithmException;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;

import org.apache.lucene.document.Document;


import org.apache.lucene.index.CorruptIndexException;

import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;

import org.apache.lucene.util.Version;
import java.io.File;
import java.io.IOException;

import org.apache.lucene.index.IndexReader;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;

import com.wsc.crawler.init.CrawlerConfig;
import com.wsc.indexer.utils.MD5CheckSum;


public class VisitedUrls{

	public static Logger log=Logger.getLogger(VisitedUrls.class.getName());
	

    public static boolean isVisited(String url,String index_Dir){
        try {
                  
            /*
             * get url to be checked and calculate its md5 hash and search with it in index.
             * if any hits found then the url is visited and return true, else not visited.
             */
        	
                  File indexDir = new File(index_Dir); //new File(conf.getVisitedUrlsDir());
                  String query =url;
                  int maxHits = 2;
       
                  query=MD5CheckSum.MD5(query.trim());
                 
                  Directory fsDir = FSDirectory.open(indexDir);
                  IndexReader reader = IndexReader.open(fsDir);
                  IndexSearcher searcher = new IndexSearcher(reader);

                  String dField = "md5";
                  Analyzer stdAn = new StandardAnalyzer(Version.LUCENE_36);
                  QueryParser parser = new QueryParser(Version.LUCENE_36,dField,stdAn);
            
                  Query q;    
                  q = parser.parse(query);         
                  
                  TopDocs hits = searcher.search(q,maxHits);
                  ScoreDoc[] scoreDocs = hits.scoreDocs;
                  /*
                   * If any hits found for given url then it is visited. 
                   */
                  for (int n = 0; n < scoreDocs.length; ++n) {
                      ScoreDoc sd = scoreDocs[n];
                    //float score = sd.score;
                      int docId = sd.doc;
                      Document d = searcher.doc(docId);
                     // String fileName = d.get("file");
                      String indxurl=d.get("url");
                      System.out.println("[INFO]:{Url Already Visited in "+VisitedUrls.class.getName()+".} ["+indxurl+"]");
                      // compare hitted url in index with parameter url.
                      return indxurl.equals(url);
                  }
                  fsDir.close();
                  reader.close();
                  searcher.close();
                  stdAn.close();
                 
        } catch (ParseException ex) {
            Logger.getLogger(VisitedUrls.class.getName()).log(Level.SEVERE, null, ex);
        } catch (NoSuchAlgorithmException ex) {
            Logger.getLogger(VisitedUrls.class.getName()).log(Level.SEVERE, null, ex);
        } catch (UnsupportedEncodingException ex) {
            Logger.getLogger(VisitedUrls.class.getName()).log(Level.SEVERE, null, ex);
        } catch (CorruptIndexException ex) {
            Logger.getLogger(VisitedUrls.class.getName()).log(Level.SEVERE, null, ex);
        } catch (IOException ex) {
            Logger.getLogger(VisitedUrls.class.getName()).log(Level.SEVERE, null, ex);
        }finally{
        	
        }
        /*
         * If no document found with given url then it return false.
         */
        return false;
    }
    
    public static boolean isVisited(String url,IndexerConfig config){
    
    	
    	boolean status=false;
    	
    	//check for this url in all indexes.
    	// if the url is indexed in any of the indexes then return true.
    	
    	log.info("Checking "+url+"in "+config.getIndex_2xx_dir());
    	status = isVisited(url,config.getIndex_2xx_dir());

    	if(status){
    		log.info("Checking is completed "+url+" is visited");
    		return true;
    	}
    	
    	log.info("Checking "+url+"in "+config.getIndex_3xx_dir());
    	status = isVisited(url,config.getIndex_3xx_dir());
    	
    	if(status){
    		log.info("Checking is completed "+url+" is visited");    	
    		return true;
    	}
    	
    	log.info("Checking "+url+"in "+config.getIndex_4xx_dir());
    	status = isVisited(url,config.getIndex_4xx_dir());
    	

    	if(status){
    		log.info("Checking is completed "+url+" is visited");
    		return true;
    	}

    	log.info("Checking "+url+"in "+config.getIndex_other_dir());
    	status = isVisited(url,config.getIndex_other_dir());
    	

    	if(status){
    		log.info("Checking is completed "+url+" is visited");
    		return true;
    	}

    	
    	return false;
    }
}